import os
from os import path
import asyncio #异步模块
import aiohttp #异步请求模块
from aiohttp.client import request
import aiofiles #异步文件读写
import csv  # CSV 读写
from aiocsv import AsyncReader, AsyncWriter #异步CSV
from lxml import etree #Xpath解析模块
import pandas as pd
import os
import re

# 创建文件夹 (如果已经创建那么忽略)
def mkdir(path):
 
	folder = os.path.exists(path)
 
	if not folder:                   #判断是否存在文件夹如果不存在则创建为文件夹
		os.makedirs(path)            #makedirs 创建文件时如果路径不存在会创建这个路径
		print ("---  new folder...  ---")
		print ("---  OK  ---")
 
	else:
		print ("---  There is this folder!  ---")

#异步协程下载HTML模块(GET方法)
async def aiodownload(url,f_path, f_name):
    async with aiohttp.ClientSession() as session:
        async with session.get(url) as resp:
            #打印数据
            content = await resp.text()
            mkdir(f_path)
            with open(f_path + '\\' + f_name + ".html", mode="w",encoding="utf-8") as f:
                f.write(content)
                print ("Complete!")
            return str(content)

#正则提取方法
#使用正则解析html函数,并返回列表
def parsingData_regex(text,pattern,g_list):
    print("start parsing data")
    object = re.compile(pattern, re.S)
    resultIter = object.finditer(text)
    content = []
    for item in resultIter:
        row = []
        for gp_item in g_list:
            row.append(item.group(gp_item))
        content.append(row)
    print(content)
    print("end parsing data")
    return content


#写入CSV文件方法
async def CSV_write(filepath,filename, content):
    print("start writing csv")
    print("csv content :",content)
    mkdir(filepath)
    async with aiofiles.open(filepath + "\\" + filename,mode="w",encoding="utf-8",newline="") as f:
        writer = AsyncWriter(f, dialect="unix")
        await writer.writerows(content)  
    print("CSV complete")
    return content

#下载并解析二级链接
async def aiodownload_parsing(url,f_path, f_name, pattern ,g_list, filepath, csv_name):
    html_content = await aiodownload(url,f_path,f_name)
    print("html content:",html_content)
    parsing_content = parsingData_regex(html_content,pattern, g_list)
    parsing_content = await CSV_write(filepath,csv_name,parsing_content)

file = r"C:\Users\Kunling\Desktop\frag_huan_2"
file1 = r"E:\Kunling"
